ANALYSIS AND PREDICTION OF CHURN¶
IMPORTING THE NECESSARY LIBRARIES¶
In [120]:
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
import matplotlib.gridspec as gridspec
import seaborn as sns
sns.set_style('darkgrid')
from sklearn.metrics import mean_squared_error
import warnings
warnings.filterwarnings('ignore')
%matplotlib inline
import os
import plotly.graph_objs as go
from plotly.offline import init_notebook_mode, iplot
import plotly.express as px
from collections import Counter
from sklearn.metrics import accuracy_score,mean_squared_error,roc_curve,roc_auc_score,classification_report,r2_score,confusion_matrix
#train test split, Grid Search CV
from sklearn.model_selection import train_test_split,cross_val_score,ShuffleSplit,GridSearchCV
from sklearn.model_selection import train_test_split
#for one hot encoding with sklearn
from sklearn.preprocessing import OneHotEncoder
#for one hot encoding with feature-engine
from feature_engine.categorical_encoders import OneHotCategoricalEncoder
In [2]:
churn = pd.read_csv("Churn_Modelling.csv")
churn.head()
Out[2]:
In [3]:
churn.tail()
Out[3]:
In [44]:
churn = churn.copy()
churn.info()
PERFORMING EDA¶
In [4]:
churn.dtypes
Out[4]:
- The dataset comprises of both numerical and catagorical variables/features
In [5]:
churn.isna().sum()
Out[5]:
- The dataset has no missing values
In [ ]:
In [7]:
# Removing the first three columns from the data set because of their irrelvance'
# churn = churn.drop(["RowNumber", "CustomerId", "Surname"], axis=1)
# churn.head(5)
After dropping the three variables from the dataset, i am now left with 11 columns¶
Statistical Summary¶
In [7]:
churn['Exited'].value_counts(normalize=True) # counting the number of customers that churn and those that didn't churn
Out[7]:
In [8]:
# Knowing the % of those that have exited and those that have not exited
print(sum(churn["Exited"]) / len(churn["Exited"]))
print(1 - sum(churn["Exited"]) / len(churn["Exited"]))
It shows that 0.2037% of customers as exited
In [45]:
churn.drop(["RowNumber", "CustomerId"], axis=1).describe().T
Out[45]:
In [ ]:
In [ ]:
In [50]:
# Checking for correlation matrix
churn.corr()
Out[50]:
- From the analysis above, it shows that the variables are not coorrelated.
In [53]:
plt.style.use("ggplot")
f,ax=plt.subplots(figsize = (15, 10))
sns.heatmap(churn.drop(["RowNumber", "CustomerId"], axis=1).corr(),
robust=True, fmt=' .1g',
linewidths=1.3,
linecolor='green',
annot=True,);
Visualizing the Dataset¶
- Countries with the less and the most common use of credit cards
- The country with the highest salary
- Visualization of credit card’s usage according to ages
- Rates of credit card’s usage according to gender
- The country with the highest credit score
- The gender with the highest credit score
- The range of age with the highest credit score
- Customers in which age range work longer with their banks
- Customers leaving the bank
- Female – Male rates
- Age distribution
- Salary distribution
In [11]:
# Visualizing the count of 'exited customers' in the dataset
plt.figure(figsize=(7,8))
sns.countplot(x='Exited', data=churn)
plt.xlabel('0: Customers still with the bank, 1: Customers exited the bank')
plt.ylabel('Count')
plt.title('Customers Churn Visualization')
plt.show()
In [54]:
plt.figure(figsize=(10,8))
sns.countplot(x="HasCrCard",
hue = "Geography",
data = churn, palette="husl");
print(churn.groupby('Geography')['HasCrCard'].sum())
In [56]:
plt.figure(figsize=(10,8))
sns.countplot(x="HasCrCard",
hue = "Exited",
data = churn);
print(churn.groupby('Geography')['HasCrCard'].sum())
In [57]:
fig = px.box(churn, x="Geography", y = "EstimatedSalary",color = 'Exited'); # Another visualization about salary effect
fig.update_layout(title_text="The country with the mean salary-With Outliers(Exited-Not Exited groups)")
fig.show();
In [62]:
plt.figure(figsize = (20,8))
plt.xticks(rotation=90)
plt.title('Credit Card Usage for Ages',color = 'blue',fontsize=15)
sns.countplot(x=churn["Age"],hue = 'HasCrCard',data=churn);
plt.xlabel('Ages')
plt.ylabel('Number of Credit Card Users');
In [63]:
fig = px.box(churn, x="HasCrCard", y = "Age", color= "Exited");
fig.update_layout(title_text = "Credit Card Usage & Age - With Outliers(Exited-Not Exited groups")
fig.show();
Credit card’s usage according to gender¶
In [66]:
fig = px.parallel_categories(churn, dimensions=['Gender', 'Geography', 'Exited'],
color="Exited",
color_continuous_scale=px.colors.sequential.Inferno,
labels={'Gender':'Gender(Female,Male)', 'Exited':'Exited(0:No,1:Yes)'})
fig.update_layout(title_text="Gender-Geography-Exited-Not Exited Schema")
fig.show();
In [68]:
fig = px.parallel_categories(churn, dimensions=['Gender','HasCrCard',"IsActiveMember", 'Exited'],
color="Exited", color_continuous_scale=px.colors.sequential.Inferno,
labels={'HasCrCard':'Has Credit Card', 'Gender':'Gender(Female,Male)', 'Exited':'Exited(0:No,1:Yes)'})
fig.update_layout(title_text="Credit Card-Gender-Exited-Not Exited Schema")
fig.show();
Highest credit score base on Geography¶
In [69]:
print(churn.groupby("Geography")["CreditScore"].mean())
fig = px.box(churn, x="Geography", y = "CreditScore",color = 'Exited');
fig.update_layout(title_text="The country with the highest credit score(mean)-With Outliers(Exited-Not Exited groups)")
fig.show();
In [ ]:
plt.figure(figsize = (14,8));
sns.catplot(x='Geography',
y = "CreditScore",
hue="Exited",
col="Gender",
aspect=1.2,height=5,
kind="swarm", data=df);
Age group with the highest credit score¶
In [73]:
plt.figure(figsize = (16,6))
plt.xticks(rotation=45)
sns.scatterplot(x=churn['Age'],y = churn["CreditScore"],hue = "Gender", data=churn);
In [72]:
plt.figure(figsize = (16,6))
plt.xticks(rotation=75)
sns.scatterplot(x=churn['Age'],y = churn["CreditScore"],hue = "Exited", data=churn);
In [77]:
plt.figure(figsize = (20,8))
plt.xticks(rotation=45)
sns.countplot(x=churn["Age"],hue = 'Exited',data=churn, palette="husl");
plt.xlabel('Age')
plt.ylabel('Number of customers (Exited or not)');
In [78]:
below_30 = churn[churn["Age"]<30]
between_30_40 = churn[(churn["Age"]>=30) & (churn["Age"]<40)]
between_40_50 = churn[(churn["Age"]>=40) & (churn["Age"]<50)]
between_50_60 = churn[(churn["Age"]>=50) & (churn["Age"]<60)]
between_60_70 = churn[(churn["Age"]>=60) & (churn["Age"]<70)]
above_70 = churn[(churn["Age"]>=70)]
k = below_30["Exited"].sum()
l = between_30_40["Exited"].sum()
m = between_40_50["Exited"].sum()
n = between_50_60["Exited"].sum()
o = between_60_70["Exited"].sum()
p = above_70["Exited"].sum()
In [80]:
f,ax = plt.subplots(figsize=(15, 15))
plt.subplot(6,1,1)
sns.countplot(x=below_30["Age"],hue = 'Exited',data=churn, palette="husl");
plt.xlabel('Age')
plt.ylabel('Customers (Exited)');
plt.xticks(rotation= 30)
plt.subplot(6,1,2)
sns.countplot(x=between_30_40["Age"],hue = 'Exited',data=churn, palette="husl");
plt.xlabel('Age')
plt.ylabel('Customers (Exited)');
plt.xticks(rotation= 30)
plt.subplot(6,1,3)
sns.countplot(x=between_40_50["Age"],hue = 'Exited',data=churn, palette="husl");
plt.xlabel('Age')
plt.ylabel('Customers (Exited)');
plt.xticks(rotation= 30);
plt.subplot(6,1,4)
sns.countplot(x=between_50_60["Age"],hue = 'Exited',data=churn, palette="husl");
plt.xlabel('Age')
plt.ylabel('Customers (Exited)');
plt.xticks(rotation= 30);
plt.subplot(6,1,5)
sns.countplot(x=between_60_70["Age"],hue = 'Exited',data=churn, palette="husl");
plt.xlabel('Age')
plt.ylabel('Customers (Exited)');
plt.xticks(rotation= 30);
plt.subplot(6,1,6)
sns.countplot(x=above_70["Age"],hue = 'Exited',data=churn, palette="husl");
plt.xlabel('Age')
plt.ylabel('Customers (Exited)');
plt.xticks(rotation= 30);
Customers leaving the bank¶
- Female – Male rates
- Age distribution
- Salary distribution
In [82]:
plt.figure(figsize = (10,6))
plt.xticks(rotation=45)
sns.barplot(x=churn['Geography'],y = churn["Exited"],hue = "Gender",data=churn, palette="husl");
plt.ylabel('Percetage of people (Exited %)');
In [83]:
plt.figure(figsize = (9,6))
plt.xticks(rotation=45)
sns.countplot(x=churn["Geography"],hue = 'Exited',data=churn, palette="husl");
plt.xlabel('Geo')
plt.ylabel('Number of customers (Exited or not)');
In [84]:
print("Total Number of People By Geography\n",churn["Geography"].value_counts())
print("Number of People Exited By Geography\n",churn[churn['Exited']==1]["Geography"].value_counts(),'\n')
print("Number of People Exited By Gender in Germany \n",churn[(churn['Exited']==1)&(churn['Geography']=='Germany')]["Gender"].value_counts())
print("Number of People Exited By Gender in France \n",churn[(churn['Exited']==1)&(churn['Geography']=='France')]["Gender"].value_counts())
print("Number of People Exited By Gender in Spain \n",churn[(churn['Exited']==1)&(churn['Geography']=='Spain')]["Gender"].value_counts())
Plotting of salary distribution¶
In [87]:
plt.figure(figsize = (10,6))
plt.xticks(rotation=75)
sns.scatterplot(x='Age',y = "EstimatedSalary",hue = "Exited",data=churn);
From the plot above, it shows that salary has no effect in exit decision
In [88]:
plt.figure(figsize = (10,6))
plt.xticks(rotation=75)
sns.scatterplot(x='Age',y = "Balance",hue = "Exited",data=churn);
Performing featuring Engineering¶
In [90]:
churn = churn.copy()
Age¶
At this stage, i will create new age groups that will help for perdiction score.
In [93]:
age_group_data = [None] * len(churn['Age'])
for i in range(len(churn['Age'])):
if churn['Age'][i] < 30:
age_group_data[i] = 'Young'
elif churn['Age'][i] >=30 and churn['Age'][i] < 40:
age_group_data[i] = 'Young-Adults'
elif churn['Age'][i] >=40 and churn['Age'][i] < 50:
age_group_data[i] = 'Adults'
elif churn['Age'][i] >=50 and churn['Age'][i] < 60:
age_group_data[i] = 'Elderly-Adults'
elif churn['Age'][i] >=60 and churn['Age'][i] < 74:
age_group_data[i] = 'Old'
else:
age_group_data[i] = 'Very-Old'
churn['age_group'] = age_group_data
Credit Score¶
I will create a boundary line for credit score
In [96]:
Credit = [None] * len(churn['CreditScore'])
for i in range(len(churn['CreditScore'])):
if churn['CreditScore'][i] < 405:
Credit[i] = 0
else:
Credit[i] = 1
churn['new_credit'] = Credit
In [97]:
churn["new_credit"].value_counts()
Out[97]:
In [100]:
sns.factorplot(x = "new_credit", y = "Exited", data = churn ,kind = "bar")
plt.xticks(rotation=75)
plt.ylabel("Exited(Precent)");
In [101]:
g = sns.factorplot(x = "age_group", y = "Exited", data = churn, kind = "bar")
plt.xticks(rotation=45)
g.set_ylabels("Exited")
plt.show()
Building ML Models¶
- Interpretation of variables
- Preparing variables for model building
1 Building dependent and undependent variables 2 Determining Train and Test sets
Building a model¶
1 Trying all models of the learning algorithms 2 Setting the parameters 3 Determining best parameters 4 Doing Cross validation 5 Finding Acury score 6 The most effective variables will be determined in each model
Comparing the models¶
1 Visualization of all models’ acury score 2 The model that gives the best results will be determined 3 Research will be done on the mathematical algorithm that creates the best model.
Creation of Dummy Variables¶
In [102]:
gender_dummies = churn.replace(to_replace={'Gender': {'Female': 0,'Male':1}})
a = pd.get_dummies(churn['Geography'], prefix = "Geo_dummy")
c = pd.get_dummies(churn['age_group'], prefix = "Age_dummy")
In [103]:
frames = [gender_dummies,a,c]
churn = pd.concat(frames, axis = 1)
churn = churn.drop(["RowNumber","Geography","Surname","CustomerId",'Age','age_group','Geography',"CreditScore"],axis = 1)
churn.head()
Out[103]:
Normalizing Variables¶
In [104]:
x = churn.drop(["Exited"],axis = 1) #Independent value
y = churn["Exited"] #Depended value
In [105]:
# data normalization with sklearn
from sklearn.preprocessing import MinMaxScaler
# fit scaler on training data
norm = MinMaxScaler().fit(x)
# transform independent data
x_norm = norm.transform(x)
In [106]:
x_train, x_test,y_train,y_test = train_test_split(x_norm,y,test_size = 0.3, random_state = 42)
Logistic Regression¶
In [116]:
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import confusion_matrix, accuracy_score
log_reg = LogisticRegression().fit(x_train,y_train)
y_pred = log_reg.predict(x_test)
log_model = (accuracy_score(y_test,y_pred)*100)
log_model
Out[116]:
In [117]:
y_probs = log_reg.predict_proba(x_test)[:,1]
y_pred = [1 if i >0.53 else 0 for i in y_probs]
log_proba_score = (accuracy_score(y_test,y_pred)*100)
print ("log score=",log_proba_score)
In [118]:
confusion_matrix(y_test,y_pred)
Out[118]:
Tuning¶
In [121]:
log_params = {"C":np.logspace(-3,3,7),
"penalty": ["l1","l2"],
"max_iter":[10,50,500,1000]} #"solver":['lbfgs', 'liblinear', 'sag', 'saga'],
log =LogisticRegression()
log_cv = GridSearchCV(log,log_params,cv = 10)
log_tuned = log_cv.fit(x_train,y_train)
log_tuned.best_params_
Out[121]:
In [122]:
log_reg_tuned = LogisticRegression(C=100,max_iter=50,penalty='l2',solver='liblinear').fit(x_train,y_train)
y_probs = log_reg.predict_proba(x_test)[:,1]
y_pred = [1 if i >0.53 else 0 for i in y_probs]
In [123]:
log_tuned_score = (accuracy_score(y_test,y_pred)*100)
print ("log tuned score=",log_tuned_score)
Confusion Matrix¶
In [124]:
lr_cm = confusion_matrix(y_test,y_pred)
lr_cm
Out[124]:
In [ ]:
Naive Bayes¶
Gaussian NB
In [125]:
from sklearn.naive_bayes import GaussianNB
nb = GaussianNB()
gnb_model = nb.fit(x_train,y_train)
gnb_model
Out[125]:
In [128]:
y_pred = gnb_model.predict(x_test)
nb_score = (accuracy_score(y_test,y_pred)*100)
nb_score
Out[128]:
Tuning¶
In [129]:
nb_params = {'var_smoothing': np.logspace(0,-9, num=100)}
nb =GaussianNB()
nb_cv = GridSearchCV(nb,nb_params,cv = 10)
nb_cv = nb_cv.fit(x_train,y_train)
nb_cv.best_params_
Out[129]:
In [130]:
nb_tuned =GaussianNB(var_smoothing=0.43287612810830584).fit(x_train,y_train)
y_pred = nb_tuned.predict(x_test)
nb_tuned = (accuracy_score(y_test,y_pred)*100)
nb_tuned
Out[130]:
Confusion Matrix¶
In [131]:
nb_cm = confusion_matrix(y_test,y_pred)
nb_cm
Out[131]:
KNN¶
In [132]:
from sklearn.neighbors import KNeighborsClassifier
knn =KNeighborsClassifier()
knn_model = knn.fit(x_train,y_train)
knn_model
Out[132]:
In [133]:
y_pred = knn_model.predict(x_test)
knn_score = (accuracy_score(y_test,y_pred)*100)
knn_score
Out[133]:
Model Tuning¶
In [134]:
knn_params = {"n_neighbors":np.arange(1,50),
"weights": ["uniform","distance"],
"metric":["euclidean","manhattan"]}
In [135]:
knn =KNeighborsClassifier()
knn_cv = GridSearchCV(knn,knn_params,cv = 10)
knn_cv = knn_cv.fit(x_train,y_train)
In [136]:
print("Best Parameters:"+str(knn_cv.best_params_))
In [137]:
knn_final =KNeighborsClassifier(n_neighbors =15,metric='manhattan',weights='distance')
knn_final = knn_final.fit(x_train,y_train)
y_pred = knn_final.predict(x_test)
knn_tuned = (accuracy_score(y_test,y_pred)*100)
knn_tuned
Out[137]:
Confusion Matrix¶
In [138]:
knn_cm = confusion_matrix(y_test,y_pred)
knn_cm
Out[138]:
SVM (Support Vector Machines)¶
In [139]:
from sklearn.svm import SVC
In [140]:
svm_model_linear = SVC(kernel='linear').fit(x_train,y_train)
svm_model_poly = SVC(kernel='poly').fit(x_train,y_train)
svm_model_rbf = SVC(kernel='rbf').fit(x_train,y_train)
In [141]:
y_pred_linear = svm_model_linear.predict(x_test)
y_pred_poly = svm_model_poly.predict(x_test)
y_pred_rbf = svm_model_rbf.predict(x_test)
In [142]:
print(accuracy_score(y_test,y_pred_linear)*100)
print(accuracy_score(y_test,y_pred_poly)*100)
print(accuracy_score(y_test,y_pred_rbf)*100)
Model Tuning¶
Polinomal kernel(poly)
In [143]:
svc_params = {"C": [1,5,10,50,100,200],
'kernel':['poly','rbf'],
"gamma": [0.001, 0.01, 0.1,0.5],}
svc = SVC()
svc_cv_model = GridSearchCV(svc,svc_params,
cv = 5,
n_jobs = -1,
verbose = 2)
svc_cv_model.fit(x_train,y_train)
print("Best Parameters:"+str(svc_cv_model.best_params_))
In [146]:
svc_tuned = SVC(kernel = "poly",C=50,gamma=0.5).fit(x_train,y_train)
In [147]:
y_pred = svc_tuned.predict(x_test)
svc_tuned_score = (accuracy_score(y_test,y_pred)*100)
svc_tuned_score
Out[147]:
In [148]:
confusion_matrix(y_test,y_pred)
Out[148]:
In [144]:
##### Radial basis function kernel(rbf)
In [150]:
svc_rbf_tuned = SVC(kernel = "rbf",C=50,gamma=0.1).fit(x_train,y_train)
y_pred = svc_rbf_tuned.predict(x_test)
In [151]:
svc_rbf_score = (accuracy_score(y_test,y_pred)*100)
svc_rbf_score
Out[151]:
Confusion Matrix¶
In [153]:
svm_cm = confusion_matrix(y_test,y_pred)
svm_cm
Out[153]:
Random Forest¶
In [154]:
from sklearn.ensemble import RandomForestClassifier
r_for = RandomForestClassifier().fit(x_train,y_train)
r_for
Out[154]:
In [155]:
y_pred = r_for.predict(x_test)
rf_score = accuracy_score(y_test,y_pred)*100
rf_score
Out[155]:
Model Tuning¶
In [156]:
rf_params = {'max_depth':list(range(1,10)),
"max_features":["log2","auto","sqrt"],
"n_estimators":[2,10,20,50,150,300],
'criterion' : ['gini','entropy'],
'min_samples_leaf' : [1,3,5,10]}
In [157]:
rf_model = RandomForestClassifier()
In [158]:
rf_cv_model = GridSearchCV(rf_model,
rf_params,
cv = 5,
n_jobs = -1)
In [159]:
rf_cv_model.fit(x_train,y_train)
rf_cv_model.best_params_
Out[159]:
In [160]:
rf_tuned = RandomForestClassifier(max_depth = 10,
criterion = 'gini',
max_features = 'log2',
min_samples_leaf = 1,
n_estimators = 150,random_state=42)
rf_tuned = rf_tuned.fit(x_train,y_train)
y_pred = rf_tuned.predict(x_test)
rf_tuned_score = (accuracy_score(y_test,y_pred)*100)
rf_tuned_score
Out[160]:
Confusion Matrix¶
In [161]:
rf_cm = confusion_matrix(y_test,y_pred)
rf_cm
Out[161]:
Gradient Boosting Machines (GBM)¶
In [163]:
from sklearn.ensemble import GradientBoostingClassifier
gbm = GradientBoostingClassifier()
gbm_model = gbm.fit(x_train,y_train)
gbm_model
Out[163]:
In [164]:
y_pred = gbm_model.predict(x_test)
gbm_score = accuracy_score(y_test,y_pred)*100
gbm_score
Out[164]:
Tuning¶
In [165]:
gbm_params = {"learning_rate" : [0.001, 0.01, 0.1, 0.2],
"n_estimators": [100,200,300,500,1000],
"max_depth": [1,3,5,10],
"min_samples_split": [1,2,5,10]}
gbm = GradientBoostingClassifier()
clf = GridSearchCV(gbm,gbm_params,verbose=0,n_jobs=-1,cv=3)
gb = clf.fit(x_train,y_train)
gb.best_params_
Out[165]:
In [166]:
gbm = GradientBoostingClassifier(n_estimators=100,min_samples_split=5,max_depth=3,learning_rate=0.2,random_state=42)
gbm.fit(x_train,y_train)
y_pred = gbm.predict(x_test)
gbm_tuned_score = accuracy_score(y_test,y_pred)*100
gbm_tuned_score
Out[166]:
Confusion Matrix¶
In [168]:
gbm_cm = confusion_matrix(y_test,y_pred)
gbm_cm
Out[168]:
XGBoost¶
In [169]:
from xgboost import XGBClassifier
xgb = XGBClassifier(n_estimators=100)
xgb_model = xgb.fit(x_train,y_train)
xgb_model
Out[169]:
In [170]:
y_pred = xgb_model.predict(x_test)
xgb_score = accuracy_score(y_test,y_pred)*100
xgb_score
Out[170]:
Tuning¶
In [171]:
xgb_params ={
'n_estimators': [50, 100, 200],
'subsample': [ 0.6, 0.8, 1.0],
'max_depth': [1,2,3,4],
'learning_rate': [0.1,0.2, 0.3, 0.4, 0.5],
"min_samples_split": [1,2,4,6]}
In [172]:
xgb = XGBClassifier()
xgb = GridSearchCV(xgb,xgb_params,verbose=0,n_jobs=-1,cv=3)
xgb = xgb.fit(x_train,y_train)
xgb.best_params_
Out[172]:
In [173]:
xgbm_cv = XGBClassifier(learning_rate=0.3,
max_depth=2,
min_samples_split=1,
n_estimators=100,
subsample=1.0,random_state=42).fit(x_train,y_train)
In [174]:
y_pred = xgbm_cv.predict(x_test)
xgbm_score = (accuracy_score(y_test,y_pred)*100)
xgbm_score
Out[174]:
Confusion Matrix¶
In [175]:
xgbm_cm = confusion_matrix(y_test,y_pred)
xgbm_cm
Out[175]:
Light GBM¶
In [176]:
from lightgbm import LGBMClassifier
lgbm = LGBMClassifier().fit(x_train,y_train)
y_pred = lgbm.predict(x_test)
In [177]:
lgbm_score = (accuracy_score(y_test,y_pred)*100)
lgbm_score
Out[177]:
Tuning¶
In [178]:
lgbm_params = {"learning_rate" : [0.001,0.01, 0.1],
"n_estimators": [100,200,300,500,1000],
"max_depth": [2,3,5,7],
"min_child_samples": [1,3,5,7]}
lgbm = LGBMClassifier()
lgbm_cv = GridSearchCV(lgbm,lgbm_params,verbose=0,n_jobs=-1,cv=5)
lgbm_cv_model = lgbm_cv.fit(x_train,y_train)
lgbm_cv_model.best_params_
Out[178]:
In [179]:
lgbm = LGBMClassifier(learning_rate=0.01,
max_depth=5,
min_child_samples=5,
n_estimators=400)
lgbm_tuned = lgbm.fit(x_train,y_train)
y_pred = lgbm_tuned.predict(x_test)
lgbm_tuned_acc = (accuracy_score(y_test,y_pred)*100)
lgbm_tuned_acc
Out[179]:
In [180]:
lgbm_cm = confusion_matrix(y_test,y_pred)
lgbm_cm
Out[180]:
Cat-Boost¶
In [181]:
from catboost import CatBoostClassifier
cat_model = CatBoostClassifier().fit(x_train,y_train)
y_pred = cat_model.predict(x_test)
In [184]:
y_pred = cat_model.predict(x_test)
catb_final_score =(accuracy_score(y_test,y_pred)*100)
catb_final_score
Out[184]:
In [185]:
catb_cm = confusion_matrix(y_test,y_pred)
catb_cm
Out[185]:
Standart Scaler Effect¶
In [186]:
## We will use also scaler for improving the score of ML algorithms
from sklearn.preprocessing import StandardScaler
scaler = StandardScaler().fit(x)
x_scaled = scaler.transform(x)
In [187]:
x_train,x_test,y_train,y_test = train_test_split(x_scaled,y,
test_size = 0.30,
random_state = 42)
KNN¶
In [188]:
knn_params = {"n_neighbors":np.arange(1,50),
"weights": ["uniform","distance"],
"metric":["euclidean","manhattan"]}
knn =KNeighborsClassifier()
knn_cv = GridSearchCV(knn,knn_params,cv = 5)
knn_cv = knn_cv.fit(x_train,y_train)
print("Best Parameters:"+str(knn_cv.best_params_))
In [189]:
knn_scaled =KNeighborsClassifier(n_neighbors =29,metric='manhattan',weights='distance')
knn_scaled = knn_scaled.fit(x_train,y_train)
y_pred = knn_scaled.predict(x_test)
knn_sscore = (accuracy_score(y_test,y_pred)*100)
knn_sscore
Out[189]:
Confusion Matrix¶
In [190]:
knn_scaled_conf = confusion_matrix(y_test,y_pred)
knn_scaled_conf
Out[190]:
Support Vector Machines¶
In [192]:
svm_scaled_linear = SVC(kernel='linear').fit(x_train,y_train)
svm_scaled_poly = SVC(kernel='poly').fit(x_train,y_train)
svm_scaled_rbf = SVC(kernel='rbf').fit(x_train,y_train)
In [193]:
y_pred_slinear = svm_scaled_linear.predict(x_test)
y_pred_spoly = svm_scaled_poly.predict(x_test)
y_pred_srbf = svm_scaled_rbf.predict(x_test)
print(accuracy_score(y_test,y_pred_slinear))
print(accuracy_score(y_test,y_pred_spoly))
print(accuracy_score(y_test,y_pred_srbf))
In [194]:
svc_params = {"C": [10,50,100,500,700],
'kernel':['poly','rbf'],
"gamma": [0.001, 0.01, 0.1]}
svc = SVC()
svc_cv_model = GridSearchCV(svc,svc_params,
cv = 5,
n_jobs = -1,
verbose = 2)
svc_cv_model.fit(x_train,y_train)
print("Best Parameters:"+str(svc_cv_model.best_params_))
In [195]:
svc_scaled = SVC(kernel = 'rbf',C = 700, gamma = 0.01)
scaled = svc_scaled.fit(x_train,y_train)
y_pred = scaled.predict(x_test)
In [196]:
svm_scaled_score = (accuracy_score(y_test,y_pred)*100)
svm_scaled_score
Out[196]:
Confusion Matrix¶
In [197]:
svc_scaled_conf = confusion_matrix(y_test,y_pred)
svc_scaled_conf
Out[197]:
Random Forest¶
In [198]:
rf_params = {'max_depth':list(range(1,11)),
"max_features":["log2","auto","sqrt"],
"n_estimators":[2,10,20,50,150,300],
'criterion' : ['gini','entropy'],
'min_samples_leaf' : [1,3,5,10]}
In [199]:
rf_model = RandomForestClassifier(random_state = 42)
rf_cv_model = GridSearchCV(rf_model,
rf_params,
cv = 5,
n_jobs = -1)
rf_cv_model.fit(x_train,y_train)
rf_cv_model.best_params_
Out[199]:
In [200]:
rf_tuned = RandomForestClassifier(max_depth = 10,
criterion = 'gini',
max_features = 'log2',
min_samples_leaf = 1,
n_estimators = 150,random_state = 42)
rf_tuned = rf_tuned.fit(x_train,y_train)
y_pred = rf_tuned.predict(x_test)
rf_scaled_score = (accuracy_score(y_test,y_pred)*100)
rf_scaled_score
Out[200]:
Confusion Matrix¶
In [201]:
rf_scaled_conf = confusion_matrix(y_test,y_pred)
rf_scaled_conf
Out[201]:
LightGBM¶
In [202]:
lgbm_params = {"learning_rate" : [0.01, 0.02,0.1],
"n_estimators": [100,200,300,500,1000],
"max_depth": [2,3,5,7],
"min_child_samples": [1,2,5,10]}
lgbm = LGBMClassifier()
lgbm_cv = GridSearchCV(lgbm,lgbm_params,verbose=0,n_jobs=-1,cv=5)
lgbm_cv_model = lgbm_cv.fit(x_train,y_train)
lgbm_cv_model.best_params_
Out[202]:
In [203]:
lgbm = LGBMClassifier(learning_rate=0.02,
max_depth=5,min_child_samples=5,
n_estimators=500,random_state = 42)
lgbm_tuned = lgbm.fit(x_train,y_train)
y_pred = lgbm_tuned.predict(x_test)
lgbm_scaled_acc = (accuracy_score(y_test,y_pred)*100)
lgbm_scaled_acc
Out[203]:
Confusion Matrix¶
In [204]:
lgbm_scaled_conf = confusion_matrix(y_test,y_pred)
lgbm_scaled_conf
Out[204]:
Logistic Regression ROC Curve¶
In [ ]:
logit_roc_auc = roc_auc_score(y_test,log_reg_tuned.predict(x_test))
fpr, tpr, tresholds = roc_curve(y_test,log_reg_tuned.predict_proba(x_test)[:,1])
plt.figure(figsize=(6,6))
plt.plot(fpr,tpr,label = "AUC (area = %0.2f)"%logit_roc_auc)
plt.plot([0,1],[0,1],"r--")
plt.xlim([0.0,1.0])
plt.ylim([0.0,1.0])
plt.xlabel("False Positive Ratio")
plt.ylabel("True Positive Ratio")
plt.title('ROC Curve');
In [205]:
from sklearn.metrics import roc_auc_score
roc_auc_score(y_test,y_probs)
Out[205]:
- As we can see our Roc curve is not much close to left top corner. This means our prediction score is a bit lover than expected.
Models Comparison¶
Confusion Matrix Comparison¶
In [206]:
fig = plt.figure(figsize=(15,15))
ax1 = fig.add_subplot(4, 4, 1) # row, column, position
ax1.set_title('Logistic Regression Classification')
ax2 = fig.add_subplot(4, 4, 2) # row, column, position
ax2.set_title('KNN Classification')
ax3 = fig.add_subplot(4, 4, 3)
ax3.set_title('SVM Classification')
ax4 = fig.add_subplot(4, 4, 4)
ax4.set_title('Naive Bayes Classification')
ax5 = fig.add_subplot(4, 4, 5)
ax5.set_title('Random Forest Classification')
ax6 = fig.add_subplot(4, 4, 6)
ax6.set_title('GBM Classification')
ax7 = fig.add_subplot(4, 4, 7)
ax7.set_title('LightGBM Classification')
ax8 = fig.add_subplot(4, 4, 8)
ax8.set_title('XGBoost Classification')
ax9 = fig.add_subplot(4, 4, 9)
ax9.set_title('CatBoost Classification')
ax10 = fig.add_subplot(4, 4, 10)
ax10.set_title('KNN Scaled Classification')
ax11 = fig.add_subplot(4,4, 11)
ax11.set_title('SVC Scaled Classification')
ax12 = fig.add_subplot(4,4, 12)
ax12.set_title('Random Forest Scaled Classification')
ax13 = fig.add_subplot(4, 4, 13)
ax13.set_title('LightGBM Scaled Classification')
sns.heatmap(data=lr_cm, annot=True, linewidth=0.7, linecolor='cyan', fmt='.0f', ax=ax1, cmap='magma')
sns.heatmap(data=knn_cm, annot=True, linewidth=0.7, linecolor='cyan', fmt='.0f', ax=ax2, cmap='magma')
sns.heatmap(data=svm_cm, annot=True, linewidth=0.7, linecolor='cyan', fmt='.0f', ax=ax3, cmap='magma')
sns.heatmap(data=nb_cm, annot=True, linewidth=0.7, linecolor='cyan', fmt='.0f', ax=ax4, cmap='magma')
sns.heatmap(data=rf_cm, annot=True, linewidth=0.7, linecolor='cyan', fmt='.0f', ax=ax5, cmap='magma')
sns.heatmap(data=gbm_cm, annot=True, linewidth=0.7, linecolor='cyan', fmt='.0f', ax=ax6, cmap='magma')
sns.heatmap(data=lgbm_cm, annot=True, linewidth=0.7, linecolor='cyan', fmt='.0f', ax=ax7, cmap='magma')
sns.heatmap(data=xgbm_cm, annot=True, linewidth=0.7, linecolor='cyan', fmt='.0f', ax=ax8, cmap='magma')
sns.heatmap(data=catb_cm, annot=True, linewidth=0.7, linecolor='cyan', fmt='.0f', ax=ax9, cmap='magma')
sns.heatmap(data=knn_scaled_conf, annot=True, linewidth=0.7, linecolor='cyan', fmt='.0f', ax=ax10, cmap='magma')
sns.heatmap(data=svc_scaled_conf, annot=True, linewidth=0.7, linecolor='cyan', fmt='.0f', ax=ax11, cmap='magma')
sns.heatmap(data=rf_scaled_conf, annot=True, linewidth=0.7, linecolor='cyan', fmt='.0f', ax=ax12, cmap='magma')
sns.heatmap(data=lgbm_scaled_conf, annot=True, linewidth=0.7, linecolor='cyan', fmt='.0f', ax=ax13, cmap='magma')
plt.show()
Model Score Comparison¶
In [207]:
indexx = ["Log","RF","KNN","SVM","NB","GBM","LightGBM","XGBoost",'CatBoost',"KNN Scaled","SVM Scaled", 'RF Scaled',"LightGBM Scaled"]
regressions = [log_tuned_score,rf_tuned_score,knn_tuned,svc_rbf_score,nb_tuned,gbm_tuned_score,
lgbm_tuned_acc,xgbm_score,catb_final_score,knn_sscore,svm_scaled_score,rf_scaled_score,lgbm_scaled_acc]
plt.figure(figsize=(12,8))
sns.barplot(x=indexx,y=regressions)
plt.xticks(rotation=45)
plt.title('Model Comparision',color = 'green',fontsize=20);
In [208]:
pie_list=regressions
labels=list(zip(indexx,regressions))
fig={
"data":[
{
"values":pie_list,
"labels":labels,
"domain": {"x": [.2, 1]},
"name": "Models-Accuracy Score",
"hoverinfo":"label+percent+name",
"hole": .4,
"type": "pie"
},],
"layout":{
"title":"Accuracy Scores",
"annotations":[
{
"font":{"size":20},
"showarrow": False,
"text": "Model Scores",
"x": 0.60,
"y": 0.50
},
]
}
}
iplot(fig)
In [ ]:
In [ ]:
Conclusion
LightGBM Boost model has the higest accuracy rate (87.1)
Cat Boost model is really successfull at catching 'true positives' of conf matrix =>>>(2356,73)(307,264)
GBM model is succesfull at catching 'false positives' conf matrix ==> (2339, 90) (297, 274)
Model tuning made quite improvement in all models.
We can experienced different scores for some models with normalization and standart scaler.
The final model is depends if we aiming catching exited members or all members(true-positive/false-positive).
In [ ]: